﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HtmlAgilityPack;
using System.Data;
using App.Core.Extensions;
using System.Text.RegularExpressions;
using Excel;
using App.Core.DB;
using System.IO;

namespace WebScrapper.Implementations.ZukaBuka
{
    public class _411CA : BaseImplementation<App.Core.Base.Model.BaseEntity>
    {
        private const String DOMAIN = "http://www.411.ca";
        private DataTable _Dt = new DataTable("Doctors");

        #region Data Table Columns
        private const String NAME = "Name";
        private const String PHONE = "Phone";
        private const String FAX = "FAX";
        private const String OTHER_TELEPHONE = "Other Telephone";
        private const String ADDRESS = "Address";            
        private const String WEB_SITE = "Web Site";
        private const String EMAIL = "Email";        
        private const String CATEGORY = "Category";
        private static String FEATURES = "Features";
        private static String REMARKS = "Remarks";
        #endregion


        public _411CA(System.Windows.Forms.RichTextBox txtStatusText) : base(txtStatusText) 
        {
            _Dt.Columns.Add(new DataColumn(CATEGORY));
            _Dt.Columns.Add(new DataColumn(NAME));
            _Dt.Columns.Add(new DataColumn(ADDRESS));
            _Dt.Columns.Add(new DataColumn(PHONE));
            _Dt.Columns.Add(new DataColumn(FAX));
            _Dt.Columns.Add(new DataColumn(OTHER_TELEPHONE));
            _Dt.Columns.Add(new DataColumn(EMAIL));
            _Dt.Columns.Add(new DataColumn(WEB_SITE));
            _Dt.Columns.Add(new DataColumn(FEATURES));
            _Dt.Columns.Add(new DataColumn(REMARKS));
        }

        public override void StartScrapping(String category)
        {
            ///Generate the Input DataTable for 411.ca
            //GenerateDataTable(category);
            //return;

            String inputExcelPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Categories.xls");
            DataTable dt = ReadExcel(inputExcelPath);
            int i = 1;
            foreach (DataRow row in dt.Rows)
            {
                String url = NullHandler.GetString(row["URL"]); //"http://www.411.ca/directory/Ontario/Ottawa/Accounting.html";
                //String str = String.Format("{0}.xls", NullHandler.GetString(row["Category"]));
                try
                {
                    //url = "http://www.411.ca/directory/Ontario/Ottawa/Ceramic-and-Tile-Contractors-and-Dealers.html";
                    ProcessSinglePage(url);                    
                    if (i % 100 == 0)
                    {
                        String filePath2 = DataTableToExcel(_Dt, null, true, String.Format("{0}_Tutul_{1}.xls", i, NullHandler.GetString(row["Category"]), i)); //DataTableToExcel(_Dt, null, true, "Baaps.xls");
                        UpdateStatusText(String.Format("Large Output File Has been created in '{0}' and clearing DataTable", filePath2));
                        _Dt.Rows.Clear();
                    }
                    else
                    {
                        //String filePath = DataTableToExcel(_Dt, true, String.Format("{0}_{1}.xls", i, NullHandler.GetString(row["Category"]))); //DataTableToExcel(_Dt, null, true, "Baaps.xls");
                        String filePath = DataTableToExcel(_Dt, null, true, String.Format("{0}_{1}.xls", i, NullHandler.GetString(row["Category"])));
                        UpdateStatusText(String.Format("Output File Has been created in '{0}'", filePath));
                    }
                    //break;
                }
                catch(Exception ex)
                {
                    UpdateStatusText(String.Format("Error: processing this URL. Error Message: {0}", ex.Message));
                }
                i++;
            }
        }

        private void GenerateDataTable(string category)
        {
            String html = base.GetAuthenticatedPageContent(category, string.Empty);
            HtmlAgilityPack.HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            //HtmlNode rootElement = doc.DocumentNode.SelectSingleNode("//div[@class='yelpanelcontent']");
            HtmlNodeCollection anchors = doc.DocumentNode.SelectNodes("//a[@class='directln']"); //rootElement.SelectNodes("a");
            DataTable dt = new DataTable();
            dt.Columns.Add("Category");
            dt.Columns.Add("URL");
            foreach (HtmlNode anchor in anchors)
            {
                DataRow row = dt.NewRow();
                row["Category"] = base.CleanUpText(anchor.InnerHtml);
                row["URL"] = String.Format("{0}{1}", DOMAIN, anchor.Attributes["href"].Value);
                dt.Rows.Add(row);
            }
            base.DataTableToExcel(dt, true, "Categories2.xls");
        }

        protected void ProcessSinglePage(string url)
        {
            UpdateStatusText(String.Format("Processing URL/Category {0}", url));
            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
            //doc.Load(@"D:\Projects\Web Scrapper\webscrapper\Trunk\Codebase\Input Htmls\ZukaBuka\www.411.ca\Accounting.html");
            String htmlText = GetAuthenticatedPageContent(url, String.Empty);
            doc.LoadHtml(htmlText);
            if (doc == null)
            {
                UpdateStatusText("Failed to load page. Attempting again...");
                ProcessSinglePage(url);
            }
            else
            {
                UpdateStatusText("Loaded Successfully");
                int pageCount = 1;
                HtmlNode bottomListCount = doc.DocumentNode.SelectSingleNode("//div[@class='botlists']");
                String[] listCounters = CleanUpText(bottomListCount.InnerText).Split(' ');
                int recordCount = 0;
                if (listCounters.Length > 0)
                    recordCount = listCounters[listCounters.Length - 1].ToInt();

                UpdateStatusText(String.Format("Total {0} record found in this Category {1}", recordCount, url));

                if (recordCount > 25)
                    pageCount = recordCount / 25 + (recordCount % 25 > 0 ? 1 : 0);

                if (pageCount > 1)
                {
                    HtmlNode pager = doc.DocumentNode.SelectNodes("//div[@class='flistnext']")[0];
                    String link = pager.SelectSingleNode("a").Attributes["href"].Value;
                    String URLFormat = link.Substring(0, link.LastIndexOf('/')) + "/{0}";
                    if (!URLFormat.ToLower().StartsWith("http://"))
                        URLFormat = DOMAIN + URLFormat;

                    for (int i = 1; i <= pageCount; i++)
                    {
                        if (i == 1)
                            ProcessDocumentNode(doc);
                        else
                        {
                            String pageURL = String.Format(URLFormat, i);
                            UpdateStatusText(String.Format("Loading Next Page from {0}", pageURL));
                            String html = GetAuthenticatedPageContent(pageURL, String.Empty);

                            doc.LoadHtml(html);
                            ProcessDocumentNode(doc);
                        }
                    }
                }
                else
                    ProcessDocumentNode(doc);
                UpdateStatusText(String.Format("Successfully Loaded Document from {0}", url));
            }
        }

        private void ProcessDocumentNode(HtmlDocument doc)
        {
            var resultDivs1 = doc.DocumentNode.SelectNodes("//div[@class='npriorblock']");
            var resultDivs2 = doc.DocumentNode.SelectNodes("//div[@class='priorblock']");
            if (resultDivs1 != null && resultDivs1.Count > 0)
            {
                foreach (HtmlNode resultDiv in resultDivs1)
                {
                    ExtractInformation(resultDiv, "n");
                }
            }
            if (resultDivs2 != null && resultDivs2.Count > 0)
            {
                foreach (HtmlNode resultDiv in resultDivs2)
                {
                    ExtractInformation(resultDiv, String.Empty);
                }
            }
        }

        private void ExtractInformation(HtmlNode resultDiv, string prefix)
        {
            DataRow row = _Dt.NewRow();
            row[NAME] = CleanUpText(resultDiv.SelectSingleNode(String.Format("div[@class='{0}priortitle']", prefix)).InnerText);
            HtmlNode addressNode = resultDiv.SelectSingleNode(String.Format("div[@class='{0}prioraddress']", prefix));
            //if (addressNode != null)
            //{
            //    HtmlNode priorAddress = addressNode.SelectNodes("div")[0].SelectSingleNode("div[@class='priorunderaddr']");
            //    row[REMARKS] = CleanUpText(priorAddress.InnerText);                
            //    row[ADDRESS] = CleanUpText(addressNode.SelectNodes("div")[0].ChildNodes[0].InnerText);
            //}

            HtmlNode hiddenAddressDiv = addressNode.SelectSingleNode("div[@class='prioraddresshid']");
            if (hiddenAddressDiv == null)
                UpdateStatusText(String.Format("Address not found for this customer. [Name: {0}]", row[NAME]));
            else
            {
                HtmlNode phone = hiddenAddressDiv.SelectSingleNode("b");
                if (phone == null)
                    UpdateStatusText("No Phone found");
                else
                    row[PHONE] = CleanUpText(phone.InnerText.Replace("Phone ", String.Empty)); //CleanUpText(childNodes[0].InnerText.Replace("Phone ", String.Empty));
                HtmlNodeCollection childNodes = hiddenAddressDiv.SelectNodes("div");
                SetFaxAndOtherPhone(row, childNodes[0]);
                //row[FAX] = CleanUpText(childNodes[0].InnerText.Replace("Fax ", String.Empty));
                row[ADDRESS] = CleanUpText(childNodes[1].InnerText.Replace("&nbsp;", String.Empty));
                row[REMARKS] = CleanUpText(childNodes[2].InnerText.Replace("&nbsp;", String.Empty));
                row[WEB_SITE] = CleanUpText(childNodes[3].InnerText.Replace("&nbsp;", String.Empty));
                row[EMAIL] = CleanUpText(childNodes[4].InnerText.Replace("&nbsp;", String.Empty));
                row[CATEGORY] = CleanUpText(childNodes[5].InnerText.Replace("&nbsp;", String.Empty).Replace("Categories:", String.Empty));
                row[FEATURES] = CleanUpText(childNodes[6].InnerText.Replace("&nbsp;", String.Empty).Replace("Features:", String.Empty));
            }
            _Dt.Rows.Add(row);
        }

        private void SetFaxAndOtherPhone(DataRow row, HtmlNode div)
        {
            if (!div.InnerHtml.IsNullOrEmpty())
            {
                List<String> phones = Regex.Split(div.InnerHtml, "<br>").ToList();
                String fax = phones.SingleOrDefault(P => P.Contains("Fax"));
                if(!fax.IsNullOrEmpty())
                    row[FAX] = fax.Replace("Fax", String.Empty);

                phones.Remove(fax);
                //if(phones.Length > 0)
                //    row[FAX] = CleanUpText(phones[0].Replace("Fax ", String.Empty));
                if (phones.Count > 0)
                {
                    StringBuilder sb = new StringBuilder();
                    for (int i = 0; i < phones.Count; i++)
                    {
                        String phoneNo = CleanUpText(phones[i]);
                        if (i == 0)
                            sb.Append(CleanUpText(phoneNo));
                        else
                        {
                            if(!phoneNo.IsNullOrEmpty())
                                sb.AppendFormat(", ", phoneNo);
                        }
                    }
                    row[OTHER_TELEPHONE] = sb.ToString();
                }
            }

        }

        public override string OutputFile(List<App.Core.Base.Model.BaseEntity> collection, bool addColumnHeader, string outputFileName)
        {
            throw new NotImplementedException();
        }
    }
}
